Setup
library(tidyverse)
Registered S3 method overwritten by 'dplyr':
method from
print.rowwise_df
[30m-- [1mAttaching packages[22m --------------------------------------- tidyverse 1.2.1 --[39m
[30m[32mv[30m [34mggplot2[30m 3.2.1 [32mv[30m [34mpurrr [30m 0.3.2
[32mv[30m [34mtibble [30m 2.1.3 [32mv[30m [34mdplyr [30m 0.8.3
[32mv[30m [34mtidyr [30m 1.0.0 [32mv[30m [34mstringr[30m 1.4.0
[32mv[30m [34mreadr [30m 1.3.1 [32mv[30m [34mforcats[30m 0.4.0[39m
[30m-- [1mConflicts[22m ------------------------------------------ tidyverse_conflicts() --
[31mx[30m [34mdplyr[30m::[32mfilter()[30m masks [34mstats[30m::filter()
[31mx[30m [34mdplyr[30m::[32mlag()[30m masks [34mstats[30m::lag()[39m
library(lubridate)
Vedh攼㸶fter pakke: 㤼㸱lubridate㤼㸲
Det f昼㸸lgende objekt er maskeret fra 㤼㸱package:base㤼㸲:
date
library(readxl)
library(data.table)
Registered S3 method overwritten by 'data.table':
method from
print.data.table
data.table 1.12.6 using 4 threads (see ?getDTthreads). Latest news: r-datatable.com
Vedh攼㸶fter pakke: 㤼㸱data.table㤼㸲
De f昼㸸lgende objekter er maskerede fra 㤼㸱package:lubridate㤼㸲:
hour, isoweek, mday, minute, month, quarter, second, wday, week, yday, year
De f昼㸸lgende objekter er maskerede fra 㤼㸱package:dplyr㤼㸲:
between, first, last
Det f昼㸸lgende objekt er maskeret fra 㤼㸱package:purrr㤼㸲:
transpose
library(tsibble)
Vedh攼㸶fter pakke: 㤼㸱tsibble㤼㸲
Det f昼㸸lgende objekt er maskeret fra 㤼㸱package:data.table㤼㸲:
key
De f昼㸸lgende objekter er maskerede fra 㤼㸱package:lubridate㤼㸲:
interval, new_interval
Det f昼㸸lgende objekt er maskeret fra 㤼㸱package:dplyr㤼㸲:
id
library(feasts)
Indl攼㸶ser kr攼㸶vet pakke: fabletools
library(tsibble)
library(feasts)
Walmart
wm_train <- fread("~/kaggle/walmart/train.csv") %>%
janitor::clean_names() %>%
mutate(date = ymd(date)) %>%
mutate_at(vars(store, dept), as.factor)
wm_test<- fread("~/kaggle/walmart/test.csv") %>%
janitor::clean_names() %>%
mutate(date = ymd(date)) %>%
mutate_at(vars(store, dept), as.factor)
feats <- fread("~/kaggle/walmart/features.csv") %>% janitor::clean_names() %>% mutate(date = ymd(date))
stores <- fread("~/kaggle/walmart/stores.csv") %>% janitor::clean_names() %>% mutate(store = as.factor(store))


_
tsib %>%
mutate(is_holiday = as.numeric(is_holiday)) %>%
as.tibble() %>%
group_by(store, dept) %>%
summarize(
cor = cor(weekly_sales, is_holiday, method = "spearman", use = "na.or.complete")
) %>%
ungroup %>%
mutate(mean_cor = mean(abs(cor), na.rm = TRUE)) %>%
ggplot(aes(x = cor)) + geom_histogram() + geom_vline(aes(xintercept = mean_cor))
train_w_var <- tsib %>%
ungroup %>%
mutate(store = as.integer(store)) %>%
left_join(feats %>% select(-is_holiday) %>% mutate(date = yearweek(date)), by = c("store", "date"))
get_correlations <- function(df) {
df %>%
corrr::correlate(method = "spearman") %>%
select(variable = rowname, cor = weekly_sales) %>%
filter(variable != "weekly_sales") %>%
as.tibble()
}
library(furrr)
plan(multiprocess)
cor_df <- train_w_var %>%
mutate_at(vars(-date, -store, -dept), replace_na, replace = 0) %>%
as.tibble() %>%
group_by(store, dept) %>%
select(-date) %>%
nest() %>%
mutate(cor = future_map(data, get_correlations, .progress = TRUE))
cor_df %>%
unnest(cor) %>%
group_by(variable) %>%
mutate(mean_val = mean(abs(cor), na.rm = TRUE)) %>%
ggplot(aes(x = cor)) + geom_histogram() + geom_vline(aes(xintercept = mean_val)) + facet_grid(~variable)
train_w_var %>%
ggplot(aes(x = mark_down1, y = weekly_sales)) + geom_point() + geom_smooth()
overall_cor <- train_w_var %>%
mutate_at(vars(-date, -store, -dept), replace_na, replace = 0) %>%
as.tibble() %>%
select(-date, -store, -dept) %>%
nest() %>%
mutate(cor = future_map(data, get_correlations, .progress = TRUE))
overall_cor %>% unnest(cor) %>% select(-data) %>% arrange(desc(abs(cor)))
rm(cor_df, feats, overall_cor,stores, train_w_var, tsib, wm_feats, wm_test, wm_train)


ex%>%
left_join(svd_var_exp_df$data[[5]] %>% mutate(store = as.integer(store))) %>%
mutate(weekly_sales = replace_na(weekly_sales, 0)) %>%
mutate(error = weekly_sales - rec) %>%
group_by(comp) %>%
summarize(
sd_error = sd(error),
sd_sales = sd(weekly_sales),
pct_error = sd_error/sd_sales,
var_exp = 1 - pct_error
)
Joining, by = c("date", "is_holiday", "store")
#Rossman
rs_tsib <- rs_trn %>%
as_tsibble(key = store, index = date)
rs_feats <- rs_tsib %>%
features(sales, list(feat_stl, feat_spectral))
rs_feats %>%
gather(key = variable, value = value, seasonal_strength_week, trend_strength, spectral_entropy) %>%
ggplot(aes(x = value)) + geom_histogram() + facet_wrap(~variable)
rs_feats %>%
mutate(ds = "Rossmann") %>%
select(-store) %>%
mutate(seas_strength = seasonal_strength_week) %>%
# bind_rows(
# wm_feats %>% mutate(ds = "Walmart") %>% select(-store) %>% mutate(seas_strength = seasonal_strength_year)
# ) %>%
gather(key = variable, value = value, seas_strength, trend_strength, spectral_entropy) %>%
ggplot(aes(x = value, fill = ds)) + geom_histogram() + facet_wrap(ds~variable)
rs_w_feat <- rs_trn %>%
left_join(rs_states) %>%
left_join(weather, by = c("state_name" = "file", "date"))
rs_cor_df <- rs_w_feat %>%
select(-day_of_week, -state_name, -state, -max_gust_speed_km_h) %>%
as.tibble() %>%
group_by(store) %>%
mutate_if(is.character, ~as.numeric(as.factor(.x))) %>%
select(-date, weekly_sales = sales) %>%
nest() %>%
mutate(cor = future_map(data, get_correlations, .progress = TRUE, ))
rs_cor_df %>%
unnest(cor) %>%
group_by(variable) %>%
mutate(mean_val = mean(abs(cor), na.rm = TRUE)) %>%
ggplot(aes(x = cor)) + geom_histogram() + geom_vline(aes(xintercept = mean_val)) + facet_wrap(~variable)
overall_cor <- rs_w_feat %>%
select(-day_of_week, -state_name, -state, -max_gust_speed_km_h) %>%
as.tibble() %>%
mutate_if(is.character, ~as.numeric(as.factor(.x))) %>%
select(-date, weekly_sales = sales) %>%
get_correlations()
overall_cor %>% arrange(desc(abs(cor)))
pcs <- rs_feats %>%
select(-store) %>%
mutate_all(~replace_na(.x, replace = mean(.x, na.rm = TRUE))) %>%
prcomp(scale=TRUE)
autoplot(pcs, loadings = TRUE, loadings.label = TRUE)
rm(rs_trn, rs_tsib, rs_w_feat, weather, rs_store, rs_states, rs_feats, rs_cor_df)
Grupo Bimbo
tictoc::tic()
bb_fact <- bb_trn[, lapply(.SD, as.factor), .SDcols = c("agencia_id", "canal_id", "ruta_sak", "cliente_id", "producto_id")]
tictoc::toc()
14.53 sec elapsed






obs <- agg[, .(cliente_id, producto_id, semana, exists = 1L)]
bt <- dcast(obs, cliente_id + producto_id ~ semana, value.var = "exists") %>%
melt(., id.vars = c("cliente_id", "producto_id"), measure.vars = c("3", "4", "5", "6", "7", "8", "9"), variable.name = "semana")
full_ds <- merge(bt[, semana := as.integer(semana)], agg, all.x = TRUE)
rm(bt, agg)
gc()
for (j in seq_len(ncol(full_ds))) {
set(full_ds, which(is.na(full_ds[[j]])),j,0)
}
rm(bb_fact, bb_test_fact)
full_ds <- full_ds[order(cliente_id, producto_id, semana)]
safe_spec <- possibly(feat_spectral, otherwise = NA)
full_ds[, .(entropy = safe_spec(demanda_uni_equil) %>% as.numeric), by = .(cliente_id, producto_id)]
plan(multiprocess)
full_ds_feat <- full_ds[, .(.(demanda_uni_equil)), by = .(cliente_id, producto_id)]
full_ds_feat[, ent := future_map(V1, ~safe_spec(.x) %>% as.numeric)]
LS0tDQp0aXRsZTogIlIgTm90ZWJvb2siDQpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sNCi0tLQ0KDQojIFNldHVwDQpgYGB7cn0NCmxpYnJhcnkodGlkeXZlcnNlKQ0KbGlicmFyeShsdWJyaWRhdGUpDQpsaWJyYXJ5KHJlYWR4bCkNCmxpYnJhcnkoZGF0YS50YWJsZSkNCmxpYnJhcnkodHNpYmJsZSkNCmxpYnJhcnkoZmVhc3RzKQ0KYGBgDQoNCiMgV2FsbWFydA0KYGBge3J9DQp3bV90cmFpbiA8LSBmcmVhZCgifi9rYWdnbGUvd2FsbWFydC90cmFpbi5jc3YiKSAlPiUNCiAgamFuaXRvcjo6Y2xlYW5fbmFtZXMoKSAlPiUNCiAgbXV0YXRlKGRhdGUgPSB5bWQoZGF0ZSkpICU+JQ0KICBtdXRhdGVfYXQodmFycyhzdG9yZSwgZGVwdCksIGFzLmZhY3RvcikNCg0Kd21fdGVzdDwtIGZyZWFkKCJ+L2thZ2dsZS93YWxtYXJ0L3Rlc3QuY3N2IikgJT4lDQogIGphbml0b3I6OmNsZWFuX25hbWVzKCkgJT4lDQogIG11dGF0ZShkYXRlID0geW1kKGRhdGUpKSAlPiUgICAgICAgICAgICAgICAgDQogIG11dGF0ZV9hdCh2YXJzKHN0b3JlLCBkZXB0KSwgYXMuZmFjdG9yKQ0KDQpmZWF0cyA8LSBmcmVhZCgifi9rYWdnbGUvd2FsbWFydC9mZWF0dXJlcy5jc3YiKSAlPiUgamFuaXRvcjo6Y2xlYW5fbmFtZXMoKSAlPiUgbXV0YXRlKGRhdGUgPSB5bWQoZGF0ZSkpDQpzdG9yZXMgPC0gZnJlYWQoIn4va2FnZ2xlL3dhbG1hcnQvc3RvcmVzLmNzdiIpICU+JSBqYW5pdG9yOjpjbGVhbl9uYW1lcygpICU+JSBtdXRhdGUoc3RvcmUgPSBhcy5mYWN0b3Ioc3RvcmUpKQ0KYGBgDQoNCmBgYHtyfQ0KdHNpYiA8LSB3bV90cmFpbiAlPiUNCiAgbXV0YXRlKGRhdGUgPSB5ZWFyd2VlayhkYXRlKSkgJT4lDQogIGFzX3RzaWJibGUoa2V5ID0gYygic3RvcmUiLCAiZGVwdCIpLCBpbmRleCA9IGRhdGUpICU+JQ0KICBncm91cF9ieV9rZXkoc3RvcmUsIGRlcHQpICU+JQ0KICB0c2liYmxlOjpmaWxsX2dhcHMod2Vla2x5X3NhbGVzID0gMCkNCg0KdHNpYiAlPiUNCiAgYXMudGliYmxlKCkgJT4lIA0KICBncm91cF9ieShzdG9yZSwgZGVwdCkgJT4lDQogIHN1bW1hcml6ZShuID0gbigpLA0KICAgICAgICAgbm9uX3plcm8gPSBzdW0od2Vla2x5X3NhbGVzID4gMCksDQogICAgICAgICBwY3Rfbm9uX3plcm8gPSBub25femVyby9uLA0KICAgICAgICAgbnVtX3NlYXMgPSA1Mi9uKSAlPiUNCiAgZ2F0aGVyKGtleSA9IG1lYXN1cmUsIHZhbCA9IHZhbHVlLCBuLCBub25femVybywgcGN0X25vbl96ZXJvLCBudW1fc2VhcykgJT4lDQogIGdncGxvdChhZXMoeCA9IHZhbHVlKSkgKyBnZW9tX2hpc3RvZ3JhbSgpICsgZmFjZXRfd3JhcCh+bWVhc3VyZSwgc2NhbGVzID0gImZyZWUiKSANCg0KbGlicmFyeShmdXR1cmUpDQpwbGFuKG11bHRpcHJvY2VzcykNCnBsYW4oc2VxdWVudGlhbCkNCg0KdGljdG9jOjp0aWMoKQ0Kd21fZmVhdHMgPC0gdHNpYiAlPiUgDQogIG11dGF0ZShuID0gbigpKSAlPiUNCiAgZmlsdGVyKG4gPiA3KSAlPiUNCiAgZmVhdHVyZXMod2Vla2x5X3NhbGVzLCBsaXN0KGZlYXRfc3RsLCBmZWF0X3NwZWN0cmFsKSkNCnRpY3RvYzo6dG9jKCkNCg0KcGNzIDwtIHdtX2ZlYXRzICU+JQ0KICBzZWxlY3QoLXN0b3JlLCAtZGVwdCkgJT4lDQogIG11dGF0ZV9hbGwofnJlcGxhY2VfbmEoLngsIHJlcGxhY2UgPSBtZWFuKC54LCBuYS5ybSA9IFRSVUUpKSkgJT4lDQogIHByY29tcChzY2FsZT1UUlVFKQ0KDQoNCmxpYnJhcnkoZ2dmb3J0aWZ5KQ0KYXV0b3Bsb3QocGNzLCBsb2FkaW5ncyA9IFRSVUUsIGxvYWRpbmdzLmxhYmVsID0gVFJVRSkNCg0Kd21fdHJhaW4gJT4lDQogIGRpc3RpbmN0KHN0b3JlLCBkZXB0KSAlPiUNCiAgbGVmdF9qb2luKHN0b3JlcykgJT4lDQogIGdyb3VwX2J5KGRlcHQsIHR5cGUpICU+JQ0KICBzdW1tYXJpemUoDQogICAgbl9kaXN0aW5jdChzdG9yZSksDQogICAgbiA9IG4oKQ0KICApDQpgYGANCg0KDQpgYGB7cn0NCndtX2ZlYXRzICU+JQ0KICBnYXRoZXIoa2V5ID0gdmFyaWFibGUsIHZhbHVlID0gdmFsdWUsIHNlYXNvbmFsX3N0cmVuZ3RoX3llYXIsIHRyZW5kX3N0cmVuZ3RoLCBzcGVjdHJhbF9lbnRyb3B5KSAlPiUNCiAgZ2dwbG90KGFlcyh4ID0gdmFsdWUpKSArIGdlb21faGlzdG9ncmFtKCkgKyBmYWNldF93cmFwKH52YXJpYWJsZSkNCmBgYA0KDQoNCl8NCmBgYHtyfQ0KdHNpYiAlPiUNCiAgbXV0YXRlKGlzX2hvbGlkYXkgPSBhcy5udW1lcmljKGlzX2hvbGlkYXkpKSAlPiUNCiAgYXMudGliYmxlKCkgJT4lDQogIGdyb3VwX2J5KHN0b3JlLCBkZXB0KSAlPiUNCiAgc3VtbWFyaXplKA0KICAgIGNvciA9IGNvcih3ZWVrbHlfc2FsZXMsIGlzX2hvbGlkYXksIG1ldGhvZCA9ICJzcGVhcm1hbiIsIHVzZSA9ICJuYS5vci5jb21wbGV0ZSIpDQogICkgJT4lDQogIHVuZ3JvdXAgJT4lDQogIG11dGF0ZShtZWFuX2NvciA9IG1lYW4oYWJzKGNvciksIG5hLnJtID0gVFJVRSkpICU+JQ0KICBnZ3Bsb3QoYWVzKHggPSBjb3IpKSArIGdlb21faGlzdG9ncmFtKCkgKyBnZW9tX3ZsaW5lKGFlcyh4aW50ZXJjZXB0ID0gbWVhbl9jb3IpKQ0KYGBgDQoNCmBgYHtyfQ0KdHJhaW5fd192YXIgPC0gdHNpYiAlPiUNCiAgdW5ncm91cCAlPiUNCiAgbXV0YXRlKHN0b3JlID0gYXMuaW50ZWdlcihzdG9yZSkpICU+JQ0KICBsZWZ0X2pvaW4oZmVhdHMgJT4lIHNlbGVjdCgtaXNfaG9saWRheSkgJT4lIG11dGF0ZShkYXRlID0geWVhcndlZWsoZGF0ZSkpLCBieSA9IGMoInN0b3JlIiwgImRhdGUiKSkNCmBgYA0KDQpgYGB7cn0NCmdldF9jb3JyZWxhdGlvbnMgPC0gZnVuY3Rpb24oZGYpIHsNCiAgZGYgJT4lIA0KICAgIGNvcnJyOjpjb3JyZWxhdGUobWV0aG9kID0gInNwZWFybWFuIikgJT4lDQogICAgc2VsZWN0KHZhcmlhYmxlID0gcm93bmFtZSwgY29yID0gd2Vla2x5X3NhbGVzKSAlPiUNCiAgICBmaWx0ZXIodmFyaWFibGUgIT0gIndlZWtseV9zYWxlcyIpICU+JQ0KICAgIGFzLnRpYmJsZSgpDQp9DQoNCmxpYnJhcnkoZnVycnIpDQpwbGFuKG11bHRpcHJvY2VzcykNCg0KY29yX2RmIDwtIHRyYWluX3dfdmFyICU+JQ0KICBtdXRhdGVfYXQodmFycygtZGF0ZSwgLXN0b3JlLCAtZGVwdCksIHJlcGxhY2VfbmEsIHJlcGxhY2UgPSAwKSAlPiUNCiAgYXMudGliYmxlKCkgJT4lDQogIGdyb3VwX2J5KHN0b3JlLCBkZXB0KSAlPiUNCiAgc2VsZWN0KC1kYXRlKSAlPiUNCiAgbmVzdCgpICU+JQ0KICBtdXRhdGUoY29yID0gZnV0dXJlX21hcChkYXRhLCBnZXRfY29ycmVsYXRpb25zLCAucHJvZ3Jlc3MgPSBUUlVFKSkNCiAgDQpjb3JfZGYgJT4lDQogIHVubmVzdChjb3IpICU+JQ0KICBncm91cF9ieSh2YXJpYWJsZSkgJT4lDQogIG11dGF0ZShtZWFuX3ZhbCA9IG1lYW4oYWJzKGNvciksIG5hLnJtID0gVFJVRSkpICU+JQ0KICBnZ3Bsb3QoYWVzKHggPSBjb3IpKSArIGdlb21faGlzdG9ncmFtKCkgKyBnZW9tX3ZsaW5lKGFlcyh4aW50ZXJjZXB0ID0gbWVhbl92YWwpKSArIGZhY2V0X2dyaWQofnZhcmlhYmxlKQ0KYGBgDQoNCmBgYHtyfQ0KdHJhaW5fd192YXIgJT4lDQogIGdncGxvdChhZXMoeCA9IG1hcmtfZG93bjEsIHkgPSB3ZWVrbHlfc2FsZXMpKSArIGdlb21fcG9pbnQoKSArIGdlb21fc21vb3RoKCkNCmBgYA0KDQpgYGB7cn0NCm92ZXJhbGxfY29yIDwtIHRyYWluX3dfdmFyICU+JQ0KICBtdXRhdGVfYXQodmFycygtZGF0ZSwgLXN0b3JlLCAtZGVwdCksIHJlcGxhY2VfbmEsIHJlcGxhY2UgPSAwKSAlPiUNCiAgYXMudGliYmxlKCkgJT4lDQogIHNlbGVjdCgtZGF0ZSwgLXN0b3JlLCAtZGVwdCkgJT4lDQogIG5lc3QoKSAlPiUNCiAgbXV0YXRlKGNvciA9IGZ1dHVyZV9tYXAoZGF0YSwgZ2V0X2NvcnJlbGF0aW9ucywgLnByb2dyZXNzID0gVFJVRSkpDQoNCm92ZXJhbGxfY29yICU+JSB1bm5lc3QoY29yKSAlPiUgc2VsZWN0KC1kYXRhKSAlPiUgYXJyYW5nZShkZXNjKGFicyhjb3IpKSkNCmBgYA0KDQoNCmBgYHtyfQ0Kcm0oY29yX2RmLCBmZWF0cywgb3ZlcmFsbF9jb3Isc3RvcmVzLCB0cmFpbl93X3ZhciwgdHNpYiwgd21fZmVhdHMsIHdtX3Rlc3QsIHdtX3RyYWluKQ0KYGBgDQoNCmBgYHtyfQ0KdHJ1bmNfc3ZkIDwtIGZ1bmN0aW9uKGRhdGFfbWF0cml4LCBjb21wb25lbnRzLCBvZ19kYXRhKSB7DQogIHogPC0gc3ZkKGRhdGFfbWF0cml4LCBudSA9IGNvbXBvbmVudHMsIG52ID0gY29tcG9uZW50cykNCiAgcyA8LSBkaWFnKHokZFsxOmNvbXBvbmVudHNdLCBjb21wb25lbnRzLCBjb21wb25lbnRzKSAjVGFrZXMgNiBsYXJnZXN0IGNvbXBvbmVudHMNCiAgcmVjIDwtICh6JHUgJSolIHMgJSolIHQoeiR2KSAlPiUgYXMuZGF0YS5mcmFtZSgpKQ0KICBwcm9jIDwtIHJlYyAlPiUgbXV0YXRlKGNvbXAgPSBjb21wb25lbnRzKSAlPiUgYmluZF9jb2xzKG9nX2RhdGEgJT4lIHNlbGVjdChkYXRlLCBpc19ob2xpZGF5KSkgJT4lIGdhdGhlcihrZXkgPSBzdG9yZSwgdmFsdWUgPSByZWMsIHN0YXJ0c193aXRoKCJWIikpICU+JSBtdXRhdGUoc3RvcmUgPSBzdHJfc3ViKHN0b3JlLCAyLCAtMSkgJT4lIGFzLmludGVnZXIpDQoNCiAgcmV0dXJuKHByb2MpDQp9DQpgYGANCg0KDQpgYGB7cn0NCndtX3RyYWluICU+JQ0KICBmaWx0ZXIoZGVwdCAlaW4lIDE6MTApICU+JQ0KICBtdXRhdGUoZG95ID0geWRheShkYXRlKSkgJT4lDQogIGdncGxvdChhZXMoeCA9IGRveSwgeSA9IHdlZWtseV9zYWxlcywgZ3JvdXAgPSBzdG9yZSkpICsgZ2VvbV9saW5lKGFscGhhID0gLjUpICsgZmFjZXRfd3JhcCh5ZWFyKGRhdGUpfmRlcHQsIG5yb3cgPSAzLCBzY2FsZXMgPSAiZnJlZV95IikNCmBgYA0KDQoNCmBgYHtyfQ0Kd21fZGV4IDwtIHdtX3RyYWluICU+JQ0KICBmaWx0ZXIoZGVwdCA9PSAxKSAlPiUNCiAgc3ByZWFkKGtleSA9IHN0b3JlLCB2YWx1ZSA9IHdlZWtseV9zYWxlcykNCg0Kd21fc3ZkIDwtIHdtX2RleCAlPiUNCiAgc2VsZWN0KC1kZXB0LCAtZGF0ZSwgLWlzX2hvbGlkYXkpICU+JQ0KICBhcy5tYXRyaXgoKQ0KDQpgYGANCg0KYGBge3J9DQp0cnVuY19zdmRfdmFyX2V4cCA8LSBmdW5jdGlvbih0cmFpbl9kZikgew0KICB3bV9kZXggPC0gdHJhaW5fZGYgJT4lDQogICAgc3ByZWFkKGtleSA9IHN0b3JlLCB2YWx1ZSA9IHdlZWtseV9zYWxlcykNCg0KICB3bV9zdmQgPC0gd21fZGV4ICU+JQ0KICAgIHNlbGVjdCgtZGF0ZSwgLWlzX2hvbGlkYXkpICU+JQ0KICAgIG11dGF0ZV9hbGwocmVwbGFjZV9uYSwgcmVwbGFjZSA9IDApICU+JQ0KICAgIGFzLm1hdHJpeCgpDQogIA0KICAxOm5jb2wod21fc3ZkKSAlPiUgbWFwX2Rmcih+dHJ1bmNfc3ZkKHdtX3N2ZCwgLngsIHdtX2RleCkpICU+JSANCiAgICBsZWZ0X2pvaW4odHJhaW5fZGYgJT4lIG11dGF0ZShzdG9yZSA9IGFzLmludGVnZXIoc3RvcmUpKSkgJT4lDQogICAgICBtdXRhdGUod2Vla2x5X3NhbGVzID0gcmVwbGFjZV9uYSh3ZWVrbHlfc2FsZXMsIDApKSAlPiUgDQogICAgbXV0YXRlKGVycm9yID0gd2Vla2x5X3NhbGVzIC0gcmVjKSAlPiUNCiAgICBncm91cF9ieShjb21wKSAlPiUNCiAgICBzdW1tYXJpemUoDQogICAgICBzZF9lcnJvciA9IHNkKGVycm9yKSwNCiAgICAgIHNkX3NhbGVzID0gc2Qod2Vla2x5X3NhbGVzKSwNCiAgICAgIHBjdF9lcnJvciA9IHNkX2Vycm9yL3NkX3NhbGVzLA0KICAgICAgdmFyX2V4cCA9IDEgLSBwY3RfZXJyb3INCiAgICApDQp9DQoNCmxpYnJhcnkoZnVycnIpDQpwbGFuKG11bHRpcHJvY2VzcykNCnN2ZF92YXJfZXhwX2RmIDwtIHdtX3RyYWluICU+JQ0KICBncm91cF9ieShkZXB0KSAlPiUNCiAgbmVzdCgpDQoNCnN2ZF92YXJfZXhwIDwtIHN2ZF92YXJfZXhwX2RmICU+JQ0KICBtdXRhdGUoDQogICAgdmFyX2V4cCA9IGZ1dHVyZV9tYXAoZGF0YSwgc2FmZWx5KHRydW5jX3N2ZF92YXJfZXhwKSwgLnByb2dyZXNzID0gVFJVRSkNCiAgKSAlPiUNCiAgdW5uZXN0KHZhcl9leHApDQoNCg0KdHQgPC0gc3ZkX3Zhcl9leHAgJT4lDQogIG11dGF0ZSgNCiAgICB0ID0gbWFwX2xnbCh2YXJfZXhwLCBpcy5udWxsKSwNCiAgICBlID0gbWFwX2xnbCh2YXJfZXhwLCBpcy50aWJibGUpLA0KICApICU+JQ0KICBmaWx0ZXIoIXQsIGUpICU+JQ0KICBzZWxlY3QoLWRhdGEpDQoNCnR0ICU+JSB1bm5lc3QodmFyX2V4cCkgJT4lDQogIGdncGxvdChhZXMoeCA9IGNvbXAsIHkgPSB2YXJfZXhwLCBncm91cCA9IGRlcHQpKSArIGdlb21fbGluZSgpICsgc2NhbGVfeV9jb250aW51b3VzKGxpbWl0cyA9IHNlcSgwLCAxKSkgKyBmYWNldF93cmFwKH5kZXB0KQ0KYGBgDQoNCmBgYHtyfQ0KICB3bV9kZXggPC0gDQpzdmRfdmFyX2V4cF9kZiRkYXRhW1s1XV0gJT4lIA0KICAgIHNwcmVhZChrZXkgPSBzdG9yZSwgdmFsdWUgPSB3ZWVrbHlfc2FsZXMpDQoNCiAgd21fc3ZkIDwtIHdtX2RleCAlPiUNCiAgICBzZWxlY3QoLWRhdGUsIC1pc19ob2xpZGF5KSAlPiUNCiAgICBtdXRhdGVfYWxsKHJlcGxhY2VfbmEsIHJlcGxhY2UgPSAwKSAlPiUNCiAgICBhcy5tYXRyaXgoKQ0KICANCiAgDQogIHN2ZCh3bV9zdmQsIDEwLCAxMCkNCiAgDQpleCA8LSAxOm5jb2wod21fc3ZkKSAlPiUgbWFwX2Rmcih+dHJ1bmNfc3ZkKHdtX3N2ZCwgLngsIHdtX2RleCkpDQoNCmV4JT4lIA0KICAgIGxlZnRfam9pbihzdmRfdmFyX2V4cF9kZiRkYXRhW1s1XV0gJT4lIG11dGF0ZShzdG9yZSA9IGFzLmludGVnZXIoc3RvcmUpKSkgJT4lIA0KICAgIG11dGF0ZSh3ZWVrbHlfc2FsZXMgPSByZXBsYWNlX25hKHdlZWtseV9zYWxlcywgMCkpICU+JSANCiAgICBtdXRhdGUoZXJyb3IgPSB3ZWVrbHlfc2FsZXMgLSByZWMpICU+JQ0KICAgIGdyb3VwX2J5KGNvbXApICU+JQ0KICAgIHN1bW1hcml6ZSgNCiAgICAgIHNkX2Vycm9yID0gc2QoZXJyb3IpLA0KICAgICAgc2Rfc2FsZXMgPSBzZCh3ZWVrbHlfc2FsZXMpLA0KICAgICAgcGN0X2Vycm9yID0gc2RfZXJyb3Ivc2Rfc2FsZXMsDQogICAgICB2YXJfZXhwID0gMSAtIHBjdF9lcnJvcg0KICAgICkNCmBgYA0KDQoNCg0KI1Jvc3NtYW4NCmBgYHtyfQ0KcnNfcGF0aCA8LSAifi9rYWdnbGUvcm9zc21hbm4vIg0KcnNfdHJuIDwtIGZyZWFkKGZpbGUucGF0aChyc19wYXRoLCAidHJhaW4uY3N2IikpICU+JSBqYW5pdG9yOjpjbGVhbl9uYW1lcygpICU+JSBtdXRhdGUoZGF0ZSA9IHltZChkYXRlKSkNCnJzX3N0b3JlIDwtIGZyZWFkKGZpbGUucGF0aChyc19wYXRoLCAic3RvcmUuY3N2IikpICU+JSBqYW5pdG9yOjpjbGVhbl9uYW1lcygpDQpyc19zdGF0ZXMgPC0gZnJlYWQoZmlsZS5wYXRoKHJzX3BhdGgsICJzdGF0ZV9uYW1lcy5jc3YiKSkgJT4lDQogIGphbml0b3I6OmNsZWFuX25hbWVzKCkgJT4lDQogIGxlZnRfam9pbihmcmVhZChmaWxlLnBhdGgocnNfcGF0aCwgInN0b3JlX3N0YXRlcy5jc3YiKSkgJT4lIGphbml0b3I6OmNsZWFuX25hbWVzKCkpDQoNCndlYXRoZXIgPC0gZnJlYWQoZmlsZS5wYXRoKHJzX3BhdGgsICJ3ZWF0aGVyLmNzdiIpLCBibGFuay5saW5lcy5za2lwPVRSVUUpICU+JSBqYW5pdG9yOjpjbGVhbl9uYW1lcygpICU+JSBtdXRhdGUoZGF0ZSA9IHltZChkYXRlKSkNCmBgYA0KDQpgYGB7cn0NCnJzX3RzaWIgPC0gcnNfdHJuICU+JQ0KICBhc190c2liYmxlKGtleSA9IHN0b3JlLCBpbmRleCA9IGRhdGUpDQoNCnJzX2ZlYXRzIDwtIHJzX3RzaWIgJT4lDQogIGZlYXR1cmVzKHNhbGVzLCBsaXN0KGZlYXRfc3RsLCBmZWF0X3NwZWN0cmFsKSkNCg0KcnNfZmVhdHMgJT4lDQogIGdhdGhlcihrZXkgPSB2YXJpYWJsZSwgdmFsdWUgPSB2YWx1ZSwgc2Vhc29uYWxfc3RyZW5ndGhfd2VlaywgdHJlbmRfc3RyZW5ndGgsIHNwZWN0cmFsX2VudHJvcHkpICU+JQ0KICBnZ3Bsb3QoYWVzKHggPSB2YWx1ZSkpICsgZ2VvbV9oaXN0b2dyYW0oKSArIGZhY2V0X3dyYXAofnZhcmlhYmxlKQ0KICANCmBgYA0KDQoNCg0KDQpgYGB7cn0NCnJzX2ZlYXRzICU+JQ0KICBtdXRhdGUoZHMgPSAiUm9zc21hbm4iKSAlPiUgDQogIHNlbGVjdCgtc3RvcmUpICU+JQ0KICBtdXRhdGUoc2Vhc19zdHJlbmd0aCA9IHNlYXNvbmFsX3N0cmVuZ3RoX3dlZWspICU+JQ0KIyAgYmluZF9yb3dzKA0KIyB3bV9mZWF0cyAlPiUgbXV0YXRlKGRzID0gIldhbG1hcnQiKSAlPiUgc2VsZWN0KC1zdG9yZSkgJT4lIG11dGF0ZShzZWFzX3N0cmVuZ3RoID0gc2Vhc29uYWxfc3RyZW5ndGhfeWVhcikNCiMgICkgJT4lDQogIGdhdGhlcihrZXkgPSB2YXJpYWJsZSwgdmFsdWUgPSB2YWx1ZSwgc2Vhc19zdHJlbmd0aCwgdHJlbmRfc3RyZW5ndGgsIHNwZWN0cmFsX2VudHJvcHkpICU+JQ0KICBnZ3Bsb3QoYWVzKHggPSB2YWx1ZSwgZmlsbCA9IGRzKSkgKyBnZW9tX2hpc3RvZ3JhbSgpICsgZmFjZXRfd3JhcChkc352YXJpYWJsZSkNCmBgYA0KDQpgYGB7cn0NCnJzX3dfZmVhdCA8LSByc190cm4gJT4lDQogIGxlZnRfam9pbihyc19zdGF0ZXMpICU+JQ0KICBsZWZ0X2pvaW4od2VhdGhlciwgYnkgPSBjKCJzdGF0ZV9uYW1lIiA9ICJmaWxlIiwgImRhdGUiKSkNCg0KcnNfY29yX2RmIDwtIHJzX3dfZmVhdCAlPiUNCiAgc2VsZWN0KC1kYXlfb2Zfd2VlaywgLXN0YXRlX25hbWUsIC1zdGF0ZSwgLW1heF9ndXN0X3NwZWVkX2ttX2gpICU+JQ0KICBhcy50aWJibGUoKSAlPiUNCiAgZ3JvdXBfYnkoc3RvcmUpICU+JQ0KICBtdXRhdGVfaWYoaXMuY2hhcmFjdGVyLCB+YXMubnVtZXJpYyhhcy5mYWN0b3IoLngpKSkgJT4lDQogIHNlbGVjdCgtZGF0ZSwgd2Vla2x5X3NhbGVzID0gc2FsZXMpICU+JQ0KICBuZXN0KCkgJT4lDQogIG11dGF0ZShjb3IgPSBmdXR1cmVfbWFwKGRhdGEsIGdldF9jb3JyZWxhdGlvbnMsIC5wcm9ncmVzcyA9IFRSVUUsICkpDQoNCnJzX2Nvcl9kZiAlPiUNCiAgdW5uZXN0KGNvcikgJT4lDQogIGdyb3VwX2J5KHZhcmlhYmxlKSAlPiUNCiAgbXV0YXRlKG1lYW5fdmFsID0gbWVhbihhYnMoY29yKSwgbmEucm0gPSBUUlVFKSkgJT4lDQogIGdncGxvdChhZXMoeCA9IGNvcikpICsgZ2VvbV9oaXN0b2dyYW0oKSArIGdlb21fdmxpbmUoYWVzKHhpbnRlcmNlcHQgPSBtZWFuX3ZhbCkpICsgZmFjZXRfd3JhcCh+dmFyaWFibGUpDQpgYGANCg0KYGBge3J9DQpvdmVyYWxsX2NvciA8LSByc193X2ZlYXQgJT4lDQogIHNlbGVjdCgtZGF5X29mX3dlZWssIC1zdGF0ZV9uYW1lLCAtc3RhdGUsIC1tYXhfZ3VzdF9zcGVlZF9rbV9oKSAlPiUNCiAgYXMudGliYmxlKCkgJT4lDQogIG11dGF0ZV9pZihpcy5jaGFyYWN0ZXIsIH5hcy5udW1lcmljKGFzLmZhY3RvcigueCkpKSAlPiUNCiAgc2VsZWN0KC1kYXRlLCB3ZWVrbHlfc2FsZXMgPSBzYWxlcykgJT4lDQogIGdldF9jb3JyZWxhdGlvbnMoKQ0KDQpvdmVyYWxsX2NvciAlPiUgYXJyYW5nZShkZXNjKGFicyhjb3IpKSkNCmBgYA0KDQpgYGB7cn0NCnBjcyA8LSByc19mZWF0cyAlPiUNCiAgc2VsZWN0KC1zdG9yZSkgJT4lDQogIG11dGF0ZV9hbGwofnJlcGxhY2VfbmEoLngsIHJlcGxhY2UgPSBtZWFuKC54LCBuYS5ybSA9IFRSVUUpKSkgJT4lDQogIHByY29tcChzY2FsZT1UUlVFKQ0KDQoNCmF1dG9wbG90KHBjcywgbG9hZGluZ3MgPSBUUlVFLCBsb2FkaW5ncy5sYWJlbCA9IFRSVUUpDQoNCmBgYA0KDQoNCmBgYHtyfQ0Kcm0ocnNfdHJuLCByc190c2liLCByc193X2ZlYXQsIHdlYXRoZXIsIHJzX3N0b3JlLCByc19zdGF0ZXMsIHJzX2ZlYXRzLCByc19jb3JfZGYpDQpgYGANCg0KDQojIEdydXBvIEJpbWJvDQpgYGB7cn0NCmJiX3RybiA8LSBmcmVhZCgifi9rYWdnbGUvYmltYm9rL3RyYWluLmNzdiIpICU+JSBqYW5pdG9yOjpjbGVhbl9uYW1lcygpDQpiYl90ZXN0IDwtIGZyZWFkKCJ+L2thZ2dsZS9iaW1ib2svdGVzdC5jc3YiKSAlPiUgamFuaXRvcjo6Y2xlYW5fbmFtZXMoKQ0KDQpiYl9mYWN0IDwtIGJiX3RyblssIGxhcHBseSguU0QsIGFzLmZhY3RvciksIC5TRGNvbHMgPSBjKCJhZ2VuY2lhX2lkIiwgImNhbmFsX2lkIiwgInJ1dGFfc2FrIiwgImNsaWVudGVfaWQiLCAicHJvZHVjdG9faWQiKV0NCmJiX3Rlc3RfZmFjdCA8LSBiYl90ZXN0WywgbGFwcGx5KC5TRCwgYXMuZmFjdG9yKSwgLlNEY29scyA9IGMoImFnZW5jaWFfaWQiLCAiY2FuYWxfaWQiLCAicnV0YV9zYWsiLCAiY2xpZW50ZV9pZCIsICJwcm9kdWN0b19pZCIpXQ0KYGBgDQoNCmBgYHtyfQ0KYmJfdHJuWywgLihjb3VudCA9IC5OKSwgYnkgPSAuKGFnZW5jaWFfaWQsIGNhbmFsX2lkLCBydXRhX3NhaywgY2xpZW50ZV9pZCwgcHJvZHVjdG9faWQpXSAlPiUNCmdncGxvdChhZXMoeCA9IGNvdW50KSkgKyBnZW9tX2hpc3RvZ3JhbSgpDQpgYGANCg0KYGBge3J9DQpiYl90cm5bLCAuKGNvdW50ID0gLk4pLCBieSA9IC4oYWdlbmNpYV9pZCwgY2xpZW50ZV9pZCwgcHJvZHVjdG9faWQsIHNlbWFuYSldICU+JQ0KICAuWywgLihjb3VudCA9IC5OKSwgYnkgPSAuKGFnZW5jaWFfaWQsIGNsaWVudGVfaWQsIHByb2R1Y3RvX2lkKV0gJT4lDQogIGdncGxvdChhZXMoeCA9IGNvdW50KSkgKyBnZW9tX2hpc3RvZ3JhbSgpDQpgYGANCg0KYGBge3J9DQpiYl90cm5bLCAuKGNvdW50ID0gLk4pLCBieSA9IC4oY2xpZW50ZV9pZCwgcHJvZHVjdG9faWQsIHNlbWFuYSldICU+JQ0KICAuWywgLihjb3VudCA9IC5OKSwgYnkgPSAuKGNsaWVudGVfaWQsIHByb2R1Y3RvX2lkKV0gJT4lDQogIGdncGxvdChhZXMoeCA9IGNvdW50KSkgKyBnZW9tX2hpc3RvZ3JhbSgpDQpgYGANCg0KDQpgYGB7cn0NCmJiX3RyblssIC4oY291bnQgPSAuTiksIGJ5ID0gLihhZ2VuY2lhX2lkLCBjbGllbnRlX2lkLCBwcm9kdWN0b19pZCwgc2VtYW5hKV0gJT4lDQogIGdncGxvdChhZXMoeCA9IGNvdW50KSkgKyBnZW9tX2hpc3RvZ3JhbSgpDQpgYGANCg0KYGBge3J9DQpiYl90ZXN0WywgLihjb3VudCA9IC5OKSwgYnkgPSAuKGFnZW5jaWFfaWQsIGNsaWVudGVfaWQsIHByb2R1Y3RvX2lkLCBzZW1hbmEpXSAlPiUNCiAgLltvcmRlcihjb3VudCwgZGVjcmVhc2luZyA9IFRSVUUpXSAlPiUNCiAgZ2dwbG90KGFlcyh4ID0gY291bnQpKSArDQogIHN0YXRfZWNkZigpDQoNCmBgYA0KDQpgYGB7cn0NCmJiX3Rlc3RbLCAuKGNvdW50ID0gLk4pLCBieSA9IC4oY2xpZW50ZV9pZCwgcHJvZHVjdG9faWQsIHNlbWFuYSldICU+JQ0KICAuW29yZGVyKGNvdW50LCBkZWNyZWFzaW5nID0gVFJVRSldICU+JQ0KICBnZ3Bsb3QoYWVzKHggPSBjb3VudCkpICsNCiAgc3RhdF9lY2RmKCkNCg0KYGBgDQoNCg0KYGBge3J9DQpiYl9mYWN0WywgbGFwcGx5KC5TRCwgbmxldmVscyksIC5TRGNvbHMgPSBjKCJjbGllbnRlX2lkIiwgInByb2R1Y3RvX2lkIiwgImFnZW5jaWFfaWQiLCAicnV0YV9zYWsiLCAiY2FuYWxfaWQiKV0NCmJiX3Rlc3RfZmFjdFssIGxhcHBseSguU0QsIG5sZXZlbHMpLCAuU0Rjb2xzID0gYygiY2xpZW50ZV9pZCIsICJwcm9kdWN0b19pZCIsICJhZ2VuY2lhX2lkIiwgInJ1dGFfc2FrIiwgImNhbmFsX2lkIildDQpgYGANCg0KYGBge3J9DQphZ2cgPC0gYmJfdHJuWywgbGFwcGx5KC5TRCwgc3VtKSwgLlNEY29scyA9IGMoInZlbnRhX3VuaV9ob3kiLCAidmVudGFfaG95IiwgImRldl91bmlfcHJveGltYSIsICJkZXZfcHJveGltYSIsICJkZW1hbmRhX3VuaV9lcXVpbCIpLCBieSA9IC4oY2xpZW50ZV9pZCwgcHJvZHVjdG9faWQsIHNlbWFuYSldDQphZ2dbb3JkZXIoY2xpZW50ZV9pZCwgcHJvZHVjdG9faWQsIHNlbWFuYSldDQpgYGANCg0KDQpgYGB7cn0NCm9icyA8LSBhZ2dbLCAuKGNsaWVudGVfaWQsIHByb2R1Y3RvX2lkLCBzZW1hbmEsIGV4aXN0cyA9IDFMKV0NCg0KYnQgPC0gZGNhc3Qob2JzLCBjbGllbnRlX2lkICsgcHJvZHVjdG9faWQgfiBzZW1hbmEsIHZhbHVlLnZhciA9ICJleGlzdHMiKSAlPiUNCiAgbWVsdCguLCBpZC52YXJzID0gYygiY2xpZW50ZV9pZCIsICJwcm9kdWN0b19pZCIpLCBtZWFzdXJlLnZhcnMgPSAgYygiMyIsICI0IiwgIjUiLCAiNiIsICI3IiwgIjgiLCAiOSIpLCB2YXJpYWJsZS5uYW1lID0gInNlbWFuYSIpDQoNCmZ1bGxfZHMgPC0gbWVyZ2UoYnRbLCBzZW1hbmEgOj0gYXMuaW50ZWdlcihzZW1hbmEpXSwgYWdnLCBhbGwueCA9IFRSVUUpDQoNCnJtKGJ0LCBhZ2cpDQpnYygpDQoNCmZvciAoaiBpbiBzZXFfbGVuKG5jb2woZnVsbF9kcykpKSB7DQogIHNldChmdWxsX2RzLCB3aGljaChpcy5uYShmdWxsX2RzW1tqXV0pKSxqLDApDQp9ICANCg0Kcm0oYmJfZmFjdCwgYmJfdGVzdF9mYWN0KQ0KDQpmdWxsX2RzIDwtIGZ1bGxfZHNbb3JkZXIoY2xpZW50ZV9pZCwgcHJvZHVjdG9faWQsIHNlbWFuYSldDQpgYGANCg0KDQpgYGB7cn0NCnNhZmVfc3BlYyA8LSBwb3NzaWJseShmZWF0X3NwZWN0cmFsLCBvdGhlcndpc2UgPSBOQSkNCmZ1bGxfZHNbLCAuKGVudHJvcHkgPSBzYWZlX3NwZWMoZGVtYW5kYV91bmlfZXF1aWwpICU+JSBhcy5udW1lcmljKSwgYnkgPSAuKGNsaWVudGVfaWQsIHByb2R1Y3RvX2lkKV0NCg0KcGxhbihtdWx0aXByb2Nlc3MpDQpmdWxsX2RzX2ZlYXQgPC0gZnVsbF9kc1ssIC4oLihkZW1hbmRhX3VuaV9lcXVpbCkpLCBieSA9IC4oY2xpZW50ZV9pZCwgcHJvZHVjdG9faWQpXQ0KZnVsbF9kc19mZWF0WywgZW50IDo9IGZ1dHVyZV9tYXAoVjEsIH5zYWZlX3NwZWMoLngpICU+JSBhcy5udW1lcmljKV0NCg0KYGBgDQoNCg==